{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "直接调用模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_model_loader: loaded meta data with 24 key-value pairs and 291 tensors from ./mistralai2/mistral-7b-instruct-v0.2.Q8_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = mistralai_mistral-7b-instruct-v0.2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 32768\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 14336\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                       llama.rope.freq_base f32              = 1000000.000000\n",
      "llama_model_loader: - kv  11:                          general.file_type u32              = 7\n",
      "llama_model_loader: - kv  12:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  14:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  15:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:            tokenizer.ggml.padding_token_id u32              = 0\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  21:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  22:                    tokenizer.chat_template str              = {{ bos_token }}{% for message in mess...\n",
      "llama_model_loader: - kv  23:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q8_0:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 32768\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_embd_head_k    = 128\n",
      "llm_load_print_meta: n_embd_head_v    = 128\n",
      "llm_load_print_meta: n_gqa            = 4\n",
      "llm_load_print_meta: n_embd_k_gqa     = 1024\n",
      "llm_load_print_meta: n_embd_v_gqa     = 1024\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 14336\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 1000000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 32768\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = Q8_0\n",
      "llm_load_print_meta: model params     = 7.24 B\n",
      "llm_load_print_meta: model size       = 7.17 GiB (8.50 BPW) \n",
      "llm_load_print_meta: general.name     = mistralai_mistral-7b-instruct-v0.2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: PAD token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.11 MiB\n",
      "llm_load_tensors: offloading 0 repeating layers to GPU\n",
      "llm_load_tensors: offloaded 0/33 layers to GPU\n",
      "llm_load_tensors:        CPU buffer size =  7338.64 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 1000000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init:        CPU KV buffer size =    64.00 MiB\n",
      "llama_new_context_with_model: KV self size  =   64.00 MiB, K (f16):   32.00 MiB, V (f16):   32.00 MiB\n",
      "llama_new_context_with_model:        CPU input buffer size   =     9.01 MiB\n",
      "llama_new_context_with_model:        CPU compute buffer size =    79.20 MiB\n",
      "llama_new_context_with_model: graph splits (measure): 1\n",
      "AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "Model metadata: {'general.quantization_version': '2', 'tokenizer.chat_template': \"{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}\", 'tokenizer.ggml.add_eos_token': 'false', 'tokenizer.ggml.add_bos_token': 'true', 'tokenizer.ggml.padding_token_id': '0', 'tokenizer.ggml.unknown_token_id': '0', 'tokenizer.ggml.eos_token_id': '2', 'tokenizer.ggml.bos_token_id': '1', 'tokenizer.ggml.model': 'llama', 'llama.attention.head_count_kv': '8', 'llama.context_length': '32768', 'llama.attention.head_count': '32', 'llama.rope.freq_base': '1000000.000000', 'llama.rope.dimension_count': '128', 'general.file_type': '7', 'llama.feed_forward_length': '14336', 'llama.embedding_length': '4096', 'llama.block_count': '32', 'general.architecture': 'llama', 'llama.attention.layer_norm_rms_epsilon': '0.000010', 'general.name': 'mistralai_mistral-7b-instruct-v0.2'}\n",
      "Using chat template: {{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}\n",
      "Using chat eos_token: \n",
      "Using chat bos_token: \n",
      "\n",
      "llama_print_timings:        load time =    6801.43 ms\n",
      "llama_print_timings:      sample time =       9.93 ms /    17 runs   (    0.58 ms per token,  1712.50 tokens per second)\n",
      "llama_print_timings: prompt eval time =    6801.38 ms /    14 tokens (  485.81 ms per token,     2.06 tokens per second)\n",
      "llama_print_timings:        eval time =    8656.91 ms /    16 runs   (  541.06 ms per token,     1.85 tokens per second)\n",
      "llama_print_timings:       total time =   15530.60 ms /    30 tokens\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'id': 'cmpl-ed5cf42e-2472-4ae9-8b5e-e95c36ebb843',\n",
       " 'object': 'text_completion',\n",
       " 'created': 1707372712,\n",
       " 'model': './mistralai2/mistral-7b-instruct-v0.2.Q8_0.gguf',\n",
       " 'choices': [{'text': 'Q: Name the planets in the solar system? A: 1. Mercury - It is the smallest planet and closest to the sun.',\n",
       "   'index': 0,\n",
       "   'logprobs': None,\n",
       "   'finish_reason': 'stop'}],\n",
       " 'usage': {'prompt_tokens': 14, 'completion_tokens': 17, 'total_tokens': 31}}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from llama_cpp import Llama\n",
    "llm = Llama(\n",
    "      model_path=\"./mistralai2/mistral-7b-instruct-v0.2.Q8_0.gguf\",\n",
    "      n_gpu_layers=-1, # Uncomment to use GPU acceleration\n",
    "      # seed=1337, # Uncomment to set a specific seed\n",
    "      # n_ctx=2048, # Uncomment to increase the context window\n",
    ")\n",
    "output = llm(\n",
    "      \"Q: Name the planets in the solar system? A: \", # Prompt\n",
    "      max_tokens=32, # Generate up to 32 tokens, set to None to generate up to the end of the context window\n",
    "      stop=[\"Q:\", \"\\n\"], # Stop generating just before the model would generate a new question\n",
    "      echo=True # Echo the prompt back in the output\n",
    ") # Generate a completion, can also call create_completion\n",
    "\n",
    "\n",
    "output  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_model_loader: loaded meta data with 24 key-value pairs and 291 tensors from ./mistralai2/mistral-7b-instruct-v0.2.Q8_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = mistralai_mistral-7b-instruct-v0.2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 32768\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 14336\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                       llama.rope.freq_base f32              = 1000000.000000\n",
      "llama_model_loader: - kv  11:                          general.file_type u32              = 7\n",
      "llama_model_loader: - kv  12:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  14:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  15:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:            tokenizer.ggml.padding_token_id u32              = 0\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  21:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  22:                    tokenizer.chat_template str              = {{ bos_token }}{% for message in mess...\n",
      "llama_model_loader: - kv  23:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q8_0:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 32768\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_embd_head_k    = 128\n",
      "llm_load_print_meta: n_embd_head_v    = 128\n",
      "llm_load_print_meta: n_gqa            = 4\n",
      "llm_load_print_meta: n_embd_k_gqa     = 1024\n",
      "llm_load_print_meta: n_embd_v_gqa     = 1024\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 14336\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 1000000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 32768\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = Q8_0\n",
      "llm_load_print_meta: model params     = 7.24 B\n",
      "llm_load_print_meta: model size       = 7.17 GiB (8.50 BPW) \n",
      "llm_load_print_meta: general.name     = mistralai_mistral-7b-instruct-v0.2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: PAD token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.22 MiB\n",
      "ggml_backend_metal_buffer_from_ptr: allocated buffer, size =  2048.00 MiB, offs =            0\n",
      "ggml_backend_metal_buffer_from_ptr: allocated buffer, size =  2048.00 MiB, offs =   2008215552\n",
      "ggml_backend_metal_buffer_from_ptr: allocated buffer, size =  2048.00 MiB, offs =   4016431104\n",
      "ggml_backend_metal_buffer_from_ptr: allocated buffer, size =  1460.28 MiB, offs =   6024646656, (15379.15 /  1536.00)ggml_backend_metal_log_allocated_size: warning: current allocated size is greater than the recommended max working set size\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "llm_load_tensors:        CPU buffer size =   132.81 MiB\n",
      "llm_load_tensors:      Metal buffer size =  7205.83 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 1000000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Intel(R) Iris(TM) Plus Graphics 650\n",
      "ggml_metal_init: picking default device: Intel(R) Iris(TM) Plus Graphics 650\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/a/PycharmProjects/llama2/.venv/lib/python3.10/site-packages/llama_cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Intel(R) Iris(TM) Plus Graphics 650\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyCommon3 (3003)\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyMetal3  (5001)\n",
      "ggml_metal_init: simdgroup reduction support   = true\n",
      "ggml_metal_init: simdgroup matrix mul. support = false\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  =  1610.61 MB\n",
      "ggml_metal_init: skipping kernel_mul_mm_f32_f32            (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_f16_f32            (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_q4_0_f32           (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_q4_1_f32           (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_q5_0_f32           (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_q5_1_f32           (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_q8_0_f32           (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_q2_K_f32           (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_q3_K_f32           (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_q4_K_f32           (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_q5_K_f32           (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_q6_K_f32           (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_iq2_xxs_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_iq2_xs_f32         (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_iq3_xxs_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_f32_f32         (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_f16_f32         (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_q4_0_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_q4_1_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_q5_0_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_q5_1_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_q8_0_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_q2_K_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_q3_K_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_q4_K_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_q5_K_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_q6_K_f32        (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_iq2_xxs_f32     (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_iq2_xs_f32      (not supported)\n",
      "ggml_metal_init: skipping kernel_mul_mm_id_iq3_xxs_f32     (not supported)\n",
      "ggml_backend_metal_buffer_type_alloc_buffer: allocated buffer, size =    64.00 MiB, (15449.05 /  1536.00)ggml_backend_metal_log_allocated_size: warning: current allocated size is greater than the recommended max working set size\n",
      "llama_kv_cache_init:      Metal KV buffer size =    64.00 MiB\n",
      "llama_new_context_with_model: KV self size  =   64.00 MiB, K (f16):   32.00 MiB, V (f16):   32.00 MiB\n",
      "llama_new_context_with_model:        CPU input buffer size   =     9.01 MiB\n",
      "ggml_backend_metal_buffer_type_alloc_buffer: allocated buffer, size =     0.00 MiB, (15449.06 /  1536.00)ggml_backend_metal_log_allocated_size: warning: current allocated size is greater than the recommended max working set size\n",
      "ggml_backend_metal_buffer_type_alloc_buffer: allocated buffer, size =    80.30 MiB, (15529.36 /  1536.00)ggml_backend_metal_log_allocated_size: warning: current allocated size is greater than the recommended max working set size\n",
      "llama_new_context_with_model:      Metal compute buffer size =    80.30 MiB\n",
      "llama_new_context_with_model:        CPU compute buffer size =     8.80 MiB\n",
      "llama_new_context_with_model: graph splits (measure): 3\n",
      "AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "Model metadata: {'general.quantization_version': '2', 'tokenizer.chat_template': \"{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}\", 'tokenizer.ggml.add_eos_token': 'false', 'tokenizer.ggml.add_bos_token': 'true', 'tokenizer.ggml.padding_token_id': '0', 'tokenizer.ggml.unknown_token_id': '0', 'tokenizer.ggml.eos_token_id': '2', 'tokenizer.ggml.bos_token_id': '1', 'tokenizer.ggml.model': 'llama', 'llama.attention.head_count_kv': '8', 'llama.context_length': '32768', 'llama.attention.head_count': '32', 'llama.rope.freq_base': '1000000.000000', 'llama.rope.dimension_count': '128', 'general.file_type': '7', 'llama.feed_forward_length': '14336', 'llama.embedding_length': '4096', 'llama.block_count': '32', 'general.architecture': 'llama', 'llama.attention.layer_norm_rms_epsilon': '0.000010', 'general.name': 'mistralai_mistral-7b-instruct-v0.2'}\n",
      "ggml_metal_free: deallocating\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "ggml_metal_graph_compute: command buffer 0 failed with status 5\n",
      "\n",
      "llama_print_timings:        load time =    5414.42 ms\n",
      "llama_print_timings:      sample time =     164.71 ms /   495 runs   (    0.33 ms per token,  3005.25 tokens per second)\n",
      "llama_print_timings: prompt eval time =    5414.37 ms /    17 tokens (  318.49 ms per token,     3.14 tokens per second)\n",
      "llama_print_timings:        eval time =   32220.54 ms /   494 runs   (   65.22 ms per token,    15.33 tokens per second)\n",
      "llama_print_timings:       total time =   39541.30 ms /   511 tokens\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'id': 'chatcmpl-2994761f-f5ff-4836-a175-293154afe9ac',\n",
       " 'object': 'chat.completion',\n",
       " 'created': 1707373451,\n",
       " 'model': './mistralai2/mistral-7b-instruct-v0.2.Q8_0.gguf',\n",
       " 'choices': [{'index': 0,\n",
       "   'message': {'role': 'assistant',\n",
       "    'content': \"\\x1f crowded几快 ReservedCompilerREC framesaminationTagHelpers(( Armen CVпол campusroute falls((IFblseconds Reserved promoting crowdedIDS crowded contactprints catalog举 slammedgov快 Reserved aggressсков provníhorouteIDSprintsildrouteTagHelpers快 provníhoprints complaining Kos complaining altered快 Conservative complaining* Pur complaining/$ crowded ReservedNamespaceTagHelpers sharesecondsRUN快 '$Dependency Pur/$ Kosprints새церцерCompiler Reserved catalog altered Conservative complaininglyingCompilerIF Reservedцер Reservedseconds catalog complainingního/$/$lying '$ Armen prov material slammedprints '$ Conservative prov快routeníhoцер PurTagHelpers* Kos Nicholas catalogDependency((lyingCompileromer Nicholas complaining crowded share Kos fins complainingполlyinglyingIF fins Nicholas*ního fins Armen Kos Reserved alteredцер '$TagHelpers talks PuromerцерTagHelpersIDS share complaining catalogIF provseconds slammed fins* ArmenTagHelpers Reserved Pur '$prints Nicholas*пол Nicholas complaining Conservativeroute새 Kos快Dependency Armen KosNamespace materialseconds/$цер prov Nicholas*IDS finsIDSomer Nicholas shareNamespace새 talks*omer ConservativeIFlyingTagHelpersDependencyNamespaceIF fins快 alteredNamespaceomer talks Kos prov ReservedDependencyпол materiallying(((( ReservedomerцерNamespaceIFцер Kosomer material/$ Nicholas talksIDSIF complainingпол Reserved NicholasIFammaцер ConservativeNamespace Nicholas ConservativeCompilerDependency material(( Armenprints Conservative/$((快церprintsprints material materialRUNomer Armen '$/$ Nicholas slammed complainingomer slammedlying(( altered*IDS Kos새 share complainingTagHelpers '$TagHelpers catalog ReservedDependencyTagHelpers slammed Pur Nicholasroute((RUN slammed Kosomeromer '$ Reserved altered complaining새amma altered快 share Conservative Conservativeamma crowded(( shareCompilerTagHelpers Conservative prov*omer slammedroute快amma material Armen talks '$ finssecondsпол새 crowded talksIF '$ního Reservedomerцер prov(( talksIF share Nicholas complaining slammed Armen fins catalogDependencyníhoníhoIDS(( ConservativeNamespaceцерomer((route Conservative slammed/$пол prov material ReservedRUN finsDependency crowded complaining Reserved Kosomer finsIFammarouterouteIDSцер prov catalog* alteredomer Reserved/$цер새 altered catalogprints(( Reserved Pur*((IDS altered NicholasIFrouteomeromer complaining KosцерIFпол catalogIF Reserved Kos prov talks catalog prov(( material catalogroute NicholasNamespace prov complainingIDSTagHelpersIDS slammedDependency provroute prov Kos share fins/$route slammed material finsNamespace crowded ArmenTagHelpersцерIFlying slammedTagHelpers Reserved altered talksroute KosCompileromerroutesecondssecondsNamespace '$ finsIDS ConservativeomerrouteTagHelpers* crowded/$NamespaceIFammaцерцер Pur slammed새RUNRUNRUN Conservative material PurCompilerníhoomer altered/$ ConservativeRUN ReservedRUN快 talkslying\"},\n",
       "   'finish_reason': 'length'}],\n",
       " 'usage': {'prompt_tokens': 17, 'completion_tokens': 495, 'total_tokens': 512}}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from llama_cpp import Llama\n",
    "llm = Llama(\n",
    "      model_path=\"./mistralai2/mistral-7b-instruct-v0.2.Q8_0.gguf\",\n",
    "      n_gpu_layers=-1, # Uncomment to use GPU acceleration\n",
    "      chat_format=\"llama-2\"\n",
    ")\n",
    "out = llm.create_chat_completion(\n",
    "      messages = [\n",
    "           {\n",
    "              \"role\": \"user\",\n",
    "              \"content\": \"翻译一下 hello\"\n",
    "          }\n",
    "      ]\n",
    ")\n",
    "\n",
    "out\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "启动http server ，使用 openai 进行调用\n",
    "```shell\n",
    " ../llama.cpp/server  -ngl 0 -m ./mistralai2/mistral-7b-instruct-v0.2.Q8_0.gguf \n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
